{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = '1'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xlnet\n",
    "import transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "xlnet.XLNetModel"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xlnet.XLNetModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.insert(0, \"/home/husein/parsing/self-attentive-parser/src\")\n",
    "sys.path.append(\"/home/husein/parsing/self-attentive-parser\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import argparse\n",
    "import itertools\n",
    "import os.path\n",
    "import time\n",
    "import shutil\n",
    "import re\n",
    "import json\n",
    "\n",
    "import torch\n",
    "import torch.optim.lr_scheduler\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "import evaluate\n",
    "import trees_newline as trees\n",
    "import vocabulary\n",
    "import nkutil\n",
    "import parse_nk_alxlnet_base as parse_nk\n",
    "tokens = parse_nk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "if parse_nk.use_cuda:\n",
    "    info = torch.load('models/en_bert_dev=74.53.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = parse_nk.NKChartParser.from_spec(info['spec'], info['state_dict'])\n",
    "bert_model = info['spec']['hparams']['bert_model']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_treebank = trees.load_trees('test.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "\n",
    "tf.compat.v1.reset_default_graph()\n",
    "sess = tf.InteractiveSession()\n",
    "sd = parser.state_dict()\n",
    "\n",
    "LABEL_VOCAB = [x[0] for x in sorted(parser.label_vocab.indices.items(), key=lambda x: x[1])]\n",
    "TAG_VOCAB = [x[0] for x in sorted(parser.tag_vocab.indices.items(), key=lambda x: x[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "position_table = tf.constant(sd['embedding.position_table'].cpu().numpy(), name=\"position_table\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_layer_norm(input, torch_name, name):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=tf.constant(sd[f'{torch_name}.b_2'].cpu().numpy(), name=f\"{name}/offset\"),\n",
    "        scale=tf.constant(sd[f'{torch_name}.a_2'].cpu().numpy(), name=f\"{name}/scale\"),\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "\n",
    "def make_heads(input, shape_bthf, shape_xtf, torch_name, name):\n",
    "    res = tf.matmul(input,\n",
    "        tf.constant(sd[torch_name].cpu().numpy().transpose((1,0,2)).reshape((512, -1)), name=f\"{name}/W\"))\n",
    "    res = tf.reshape(res, shape_bthf)\n",
    "    res = tf.transpose(res, (0,2,1,3)) # batch x num_heads x time x feat\n",
    "    res = tf.reshape(res, shape_xtf) # _ x time x feat\n",
    "    return res\n",
    "\n",
    "def make_attention(input, nonpad_ids, dim_flat, dim_padded, valid_mask, torch_name, name):\n",
    "    input_flat = tf.scatter_nd(indices=nonpad_ids[:, None], updates=input, shape=tf.concat([dim_flat, tf.shape(input)[1:]], axis=0))\n",
    "    input_flat_dat, input_flat_pos = tf.split(input_flat, 2, axis=-1)\n",
    "\n",
    "    shape_bthf = tf.concat([dim_padded, [8, -1]], axis=0)\n",
    "    shape_bhtf = tf.convert_to_tensor([dim_padded[0], 8, dim_padded[1], -1])\n",
    "    shape_xtf = tf.convert_to_tensor([dim_padded[0] * 8, dim_padded[1], -1])\n",
    "    shape_xf = tf.concat([dim_flat, [-1]], axis=0)\n",
    "\n",
    "    qs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_qs1', f'{name}/q_dat')\n",
    "    ks1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_ks1', f'{name}/k_dat')\n",
    "    vs1 = make_heads(input_flat_dat, shape_bthf, shape_xtf, f'{torch_name}.w_vs1', f'{name}/v_dat')\n",
    "    qs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_qs2', f'{name}/q_pos')\n",
    "    ks2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_ks2', f'{name}/k_pos')\n",
    "    vs2 = make_heads(input_flat_pos, shape_bthf, shape_xtf, f'{torch_name}.w_vs2', f'{name}/v_pos')\n",
    "\n",
    "    qs = tf.concat([qs1, qs2], axis=-1)\n",
    "    ks = tf.concat([ks1, ks2], axis=-1)\n",
    "    attn_logits = tf.matmul(qs, ks, transpose_b=True) / (1024 ** 0.5)\n",
    "\n",
    "    attn_mask = tf.reshape(tf.tile(valid_mask, [1,8*dim_padded[1]]), tf.shape(attn_logits))\n",
    "    # TODO(nikita): use tf.where and -float('inf') here?\n",
    "    attn_logits -= 1e10 * tf.to_float(~attn_mask)\n",
    "\n",
    "    attn = tf.nn.softmax(attn_logits)\n",
    "\n",
    "    attended_dat_raw = tf.matmul(attn, vs1)\n",
    "    attended_dat_flat = tf.reshape(tf.transpose(tf.reshape(attended_dat_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_dat = tf.gather(attended_dat_flat, nonpad_ids)\n",
    "    attended_pos_raw = tf.matmul(attn, vs2)\n",
    "    attended_pos_flat = tf.reshape(tf.transpose(tf.reshape(attended_pos_raw, shape_bhtf), (0,2,1,3)), shape_xf)\n",
    "    attended_pos = tf.gather(attended_pos_flat, nonpad_ids)\n",
    "\n",
    "    out_dat = tf.matmul(attended_dat, tf.constant(sd[f'{torch_name}.proj1.weight'].cpu().numpy().transpose()))\n",
    "    out_pos = tf.matmul(attended_pos, tf.constant(sd[f'{torch_name}.proj2.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_dense_relu_dense(input, torch_name, torch_type, name):\n",
    "    # TODO: use name\n",
    "    mul1 = tf.matmul(input, tf.constant(sd[f'{torch_name}.w_1{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul1b = tf.nn.bias_add(mul1, tf.constant(sd[f'{torch_name}.w_1{torch_type}.bias'].cpu().numpy()))\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, tf.constant(sd[f'{torch_name}.w_2{torch_type}.weight'].cpu().numpy().transpose()))\n",
    "    mul2b = tf.nn.bias_add(mul2, tf.constant(sd[f'{torch_name}.w_2{torch_type}.bias'].cpu().numpy()))\n",
    "    return mul2b\n",
    "\n",
    "def make_ff(input, torch_name, name):\n",
    "    # TODO: use name\n",
    "    input_dat, input_pos = tf.split(input, 2, axis=-1)\n",
    "    out_dat = make_dense_relu_dense(input_dat, torch_name, 'c', name=\"TODO_dat\")\n",
    "    out_pos = make_dense_relu_dense(input_pos, torch_name, 'p', name=\"TODO_pos\")\n",
    "    out = tf.concat([out_dat, out_pos], -1)\n",
    "    return make_layer_norm(input + out, f'{torch_name}.layer_norm', f'{name}/layer_norm')\n",
    "\n",
    "def make_stacks(input, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks):\n",
    "    res = input\n",
    "    for i in range(num_stacks):\n",
    "        res = make_attention(res, nonpad_ids, dim_flat, dim_padded, valid_mask, f'encoder.attn_{i}', name=f'attn_{i}')\n",
    "        res = make_ff(res, f'encoder.ff_{i}', name=f'ff_{i}')\n",
    "    return res\n",
    "\n",
    "def make_layer_norm_with_constants(input, constants):\n",
    "    # TODO(nikita): The epsilon here isn't quite the same as in pytorch\n",
    "    # The pytorch code adds eps=1e-3 to the standard deviation, while this\n",
    "    # tensorflow code adds eps=1e-6 to the variance.\n",
    "    # However, the resulting mismatch in floating-point values does not seem to\n",
    "    # translate to any noticable changes in the parser's tree output\n",
    "    mean, variance = tf.nn.moments(input, [1], keep_dims=True)\n",
    "    return tf.nn.batch_normalization(\n",
    "        input,\n",
    "        mean, variance,\n",
    "        offset=constants[0],\n",
    "        scale=constants[1],\n",
    "        variance_epsilon=1e-6)\n",
    "\n",
    "def make_flabel_with_constants(input, constants):\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='flabel')\n",
    "    return mul2b\n",
    "\n",
    "def make_ftag(input):\n",
    "    constants = (\n",
    "        tf.constant(sd['f_tag.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_tag.1.b_2'].cpu().numpy(), name=\"tag/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_tag.1.a_2'].cpu().numpy(), name=\"tag/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_tag.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_tag.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "    mul1 = tf.matmul(input, constants[0])\n",
    "    mul1b = tf.nn.bias_add(mul1, constants[1])\n",
    "    mul1b = make_layer_norm_with_constants(mul1b, constants[2:4])\n",
    "    mul1b = tf.nn.relu(mul1b)\n",
    "    mul2 = tf.matmul(mul1b, constants[4])\n",
    "    mul2b = tf.nn.bias_add(mul2, constants[5], name='ftag')\n",
    "    return mul2b\n",
    "\n",
    "def make_flabel_constants():\n",
    "    return (\n",
    "        tf.constant(sd['f_label.0.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.0.bias'].cpu().numpy()),\n",
    "        tf.constant(sd['f_label.1.b_2'].cpu().numpy(), name=\"label/layer_norm/offset\"),\n",
    "        tf.constant(sd['f_label.1.a_2'].cpu().numpy(), name=\"label/layer_norm/scale\"),\n",
    "        tf.constant(sd['f_label.3.weight'].cpu().numpy().transpose()),\n",
    "        tf.constant(sd['f_label.3.bias'].cpu().numpy()),\n",
    "    )\n",
    "\n",
    "def make_network():\n",
    "    # batch x num_subwords\n",
    "    input_ids = tf.placeholder(shape=(None, None), dtype=tf.int32, name='input_ids')\n",
    "    word_end_mask = tf.placeholder(shape=(None, None), dtype=tf.int32, name='word_end_mask')\n",
    "    input_dat, nonpad_ids, dim_flat, dim_padded, valid_mask, lengths, bert_features = make_bert(input_ids, word_end_mask)\n",
    "    input_pos_flat = tf.tile(position_table[:dim_padded[1]], [dim_padded[0], 1])\n",
    "    input_pos = tf.gather(input_pos_flat, nonpad_ids)\n",
    "\n",
    "    input_joint = tf.concat([input_dat, input_pos], -1)\n",
    "    input_joint = make_layer_norm(input_joint, 'embedding.layer_norm', 'embedding/layer_norm')\n",
    "\n",
    "    word_out = make_stacks(input_joint, nonpad_ids, dim_flat, dim_padded, valid_mask, num_stacks=parser.spec['hparams']['num_layers'])\n",
    "    word_out = tf.concat([word_out[:, 0::2], word_out[:, 1::2]], -1)\n",
    "\n",
    "    # part-of-speech predictions\n",
    "    ftag = make_ftag(word_out)\n",
    "    tags_packed = tf.argmax(ftag, axis=-1)\n",
    "    tags = tf.reshape(\n",
    "        tf.scatter_nd(indices=nonpad_ids[:, None], updates=tags_packed, shape=dim_flat),\n",
    "        dim_padded\n",
    "        )\n",
    "    tags = tf.identity(tags, name=\"tags\")\n",
    "\n",
    "    fp_out = tf.concat([word_out[:-1,:512], word_out[1:,512:]], -1)\n",
    "\n",
    "    fp_start_idxs = tf.cumsum(lengths, exclusive=True)\n",
    "    fp_end_idxs = tf.cumsum(lengths) - 1 # the number of fenceposts is 1 less than the number of words\n",
    "\n",
    "    fp_end_idxs_uneven = fp_end_idxs - tf.convert_to_tensor([1, 0])\n",
    "\n",
    "    # Have to make these outside tf.map_fn for model compression to work\n",
    "    constants = make_flabel_constants()\n",
    "\n",
    "    def to_map(start_and_end):\n",
    "        start, end = start_and_end\n",
    "        fp = fp_out[start:end]\n",
    "        flabel = make_flabel_with_constants(tf.reshape(fp[None,:,:] - fp[:,None,:], (-1, 1024)), constants)\n",
    "        actual_chart_size = end-start\n",
    "        flabel = tf.reshape(flabel, [actual_chart_size, actual_chart_size, -1])\n",
    "        amount_to_pad = dim_padded[1] - actual_chart_size\n",
    "        # extra padding on the label dimension is for the not-a-constituent label,\n",
    "        # which always has a score of 0\n",
    "        flabel = tf.pad(flabel, [[0, amount_to_pad], [0, amount_to_pad], [1, 0]])\n",
    "        return flabel\n",
    "\n",
    "    charts = tf.map_fn(to_map, (fp_start_idxs, fp_end_idxs), dtype=(tf.float32))\n",
    "    charts = tf.identity(charts, name=\"charts\")\n",
    "\n",
    "    return input_ids, word_end_mask, charts, tags, bert_features\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_tf_xlnet_to_pytorch_map(model, config, tf_weights = None):\n",
    "    \"\"\" A map of modules from TF to PyTorch.\n",
    "        I use a map to keep the PyTorch model as\n",
    "        identical to the original PyTorch model as possible.\n",
    "    \"\"\"\n",
    "\n",
    "    tf_to_pt_map = {}\n",
    "\n",
    "    if hasattr(model, 'transformer'):\n",
    "        if hasattr(model, 'lm_loss'):\n",
    "            # We will load also the output bias\n",
    "            tf_to_pt_map['model/lm_loss/bias'] = model.lm_loss.bias\n",
    "        if (\n",
    "            hasattr(model, 'sequence_summary')\n",
    "            and 'model/sequnece_summary/summary/kernel' in tf_weights\n",
    "        ):\n",
    "            # We will load also the sequence summary\n",
    "            tf_to_pt_map[\n",
    "                'model/sequnece_summary/summary/kernel'\n",
    "            ] = model.sequence_summary.summary.weight\n",
    "            tf_to_pt_map[\n",
    "                'model/sequnece_summary/summary/bias'\n",
    "            ] = model.sequence_summary.summary.bias\n",
    "        if (\n",
    "            hasattr(model, 'logits_proj')\n",
    "            and config.finetuning_task is not None\n",
    "            and 'model/regression_{}/logit/kernel'.format(\n",
    "                config.finetuning_task\n",
    "            )\n",
    "            in tf_weights\n",
    "        ):\n",
    "            tf_to_pt_map[\n",
    "                'model/regression_{}/logit/kernel'.format(\n",
    "                    config.finetuning_task\n",
    "                )\n",
    "            ] = model.logits_proj.weight\n",
    "            tf_to_pt_map[\n",
    "                'model/regression_{}/logit/bias'.format(config.finetuning_task)\n",
    "            ] = model.logits_proj.bias\n",
    "\n",
    "        # Now load the rest of the transformer\n",
    "        model = model.transformer\n",
    "\n",
    "    # Embeddings and output\n",
    "    tf_to_pt_map.update(\n",
    "        {\n",
    "            'model/transformer/word_embedding/lookup_table': model.word_embedding.weight,\n",
    "            'model/transformer/word_embedding/lookup_table_2': model.word_embedding2.weight,\n",
    "            'model/transformer/mask_emb/mask_emb': model.mask_emb,\n",
    "        }\n",
    "    )\n",
    "\n",
    "    # Transformer blocks\n",
    "    for i, b in enumerate(model.layer):\n",
    "        layer_str = 'model/transformer/layer_shared/'\n",
    "        tf_to_pt_map.update(\n",
    "            {\n",
    "                layer_str\n",
    "                + 'rel_attn/LayerNorm/gamma': b.rel_attn.layer_norm.weight,\n",
    "                layer_str\n",
    "                + 'rel_attn/LayerNorm/beta': b.rel_attn.layer_norm.bias,\n",
    "                layer_str + 'rel_attn/o/kernel': b.rel_attn.o,\n",
    "                layer_str + 'rel_attn/q/kernel': b.rel_attn.q,\n",
    "                layer_str + 'rel_attn/k/kernel': b.rel_attn.k,\n",
    "                layer_str + 'rel_attn/r/kernel': b.rel_attn.r,\n",
    "                layer_str + 'rel_attn/v/kernel': b.rel_attn.v,\n",
    "                layer_str + 'ff/LayerNorm/gamma': b.ff.layer_norm.weight,\n",
    "                layer_str + 'ff/LayerNorm/beta': b.ff.layer_norm.bias,\n",
    "                layer_str + 'ff/layer_1/kernel': b.ff.layer_1.weight,\n",
    "                layer_str + 'ff/layer_1/bias': b.ff.layer_1.bias,\n",
    "                layer_str + 'ff/layer_2/kernel': b.ff.layer_2.weight,\n",
    "                layer_str + 'ff/layer_2/bias': b.ff.layer_2.bias,\n",
    "            }\n",
    "        )\n",
    "\n",
    "    # Relative positioning biases\n",
    "    if config.untie_r:\n",
    "        r_r_list = []\n",
    "        r_w_list = []\n",
    "        r_s_list = []\n",
    "        seg_embed_list = []\n",
    "        for b in model.layer:\n",
    "            r_r_list.append(b.rel_attn.r_r_bias)\n",
    "            r_w_list.append(b.rel_attn.r_w_bias)\n",
    "            r_s_list.append(b.rel_attn.r_s_bias)\n",
    "            seg_embed_list.append(b.rel_attn.seg_embed)\n",
    "    else:\n",
    "        r_r_list = [model.r_r_bias]\n",
    "        r_w_list = [model.r_w_bias]\n",
    "        r_s_list = [model.r_s_bias]\n",
    "        seg_embed_list = [model.seg_embed]\n",
    "    tf_to_pt_map.update(\n",
    "        {\n",
    "            'model/transformer/r_r_bias': r_r_list,\n",
    "            'model/transformer/r_w_bias': r_w_list,\n",
    "            'model/transformer/r_s_bias': r_s_list,\n",
    "            'model/transformer/seg_embed': seg_embed_list,\n",
    "        }\n",
    "    )\n",
    "    return tf_to_pt_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_bert(input_ids, word_end_mask):\n",
    "    # We can derive input_mask from either input_ids or word_end_mask\n",
    "    input_mask = (1 - tf.cumprod(1 - word_end_mask, axis=-1, reverse=True))\n",
    "    input_mask = (1 - input_mask)\n",
    "    # input_mask = word_end_mask\n",
    "    token_type_ids = tf.zeros_like(input_ids)\n",
    "    bert_model = make_bert_instance(input_ids, input_mask, token_type_ids)\n",
    "\n",
    "    bert_features = bert_model.get_sequence_output()\n",
    "    print(bert_features)\n",
    "    bert_features = tf.transpose(bert_features, perm=[1, 0, 2])\n",
    "    bert_features_packed = tf.gather(\n",
    "        tf.reshape(bert_features, [-1, int(bert_features.shape[-1])]),\n",
    "        tf.to_int32(tf.where(tf.reshape(word_end_mask, (-1,))))[:,0])\n",
    "    projected_annotations = tf.matmul(\n",
    "        bert_features_packed,\n",
    "        tf.constant(sd['project_bert.weight'].cpu().numpy().transpose()))\n",
    "\n",
    "    # input_mask is over subwords, whereas valid_mask is over words\n",
    "    sentence_lengths = tf.reduce_sum(word_end_mask, -1)\n",
    "    valid_mask = (tf.range(tf.reduce_max(sentence_lengths))[None,:] < sentence_lengths[:, None])\n",
    "    dim_padded = tf.shape(valid_mask)[:2]\n",
    "    mask_flat = tf.reshape(valid_mask, (-1,))\n",
    "    dim_flat = tf.shape(mask_flat)[:1]\n",
    "    nonpad_ids = tf.to_int32(tf.where(mask_flat)[:,0])\n",
    "\n",
    "    return projected_annotations, nonpad_ids, dim_flat, dim_padded, valid_mask, sentence_lengths, bert_features\n",
    "\n",
    "def make_bert_instance(input_ids, input_mask, token_type_ids):\n",
    "    # Transfer BERT config into tensorflow implementation\n",
    "    import json\n",
    "    \n",
    "    input_ids = tf.transpose(input_ids, [1, 0])\n",
    "    token_type_ids = tf.transpose(token_type_ids, [1, 0])\n",
    "    input_mask = tf.transpose(input_mask, [1, 0])\n",
    "\n",
    "    config = xlnet.XLNetConfig(json_path = 'src/alxlnet-base/config-old.json')\n",
    "\n",
    "    kwargs = dict(\n",
    "          is_training=False,\n",
    "          use_tpu=False,\n",
    "          use_bfloat16=False,\n",
    "          dropout=0,\n",
    "          dropatt=0,\n",
    "          init='normal',\n",
    "          init_range=0.1,\n",
    "          init_std=0.02,\n",
    "          clamp_len=-1)\n",
    "    \n",
    "    xlnet_parameters = xlnet.RunConfig(**kwargs)\n",
    "    model = xlnet.XLNetModel(\n",
    "        xlnet_config=config,\n",
    "        run_config=xlnet_parameters,\n",
    "        input_ids=input_ids,\n",
    "        seg_ids=token_type_ids,\n",
    "        input_mask=tf.cast(input_mask, tf.float32))\n",
    "    \n",
    "    mapping = build_tf_xlnet_to_pytorch_map(parser.bert, parser.bert.config)\n",
    "    \n",
    "    bert_variables = [v for v in tf.get_collection('variables')]\n",
    "    print([v.name for v in tf.get_collection('variables')])\n",
    "    tf.variables_initializer(bert_variables).run()\n",
    "    \n",
    "    for i in range(len(bert_variables)):\n",
    "        variable = bert_variables[i]\n",
    "        name = variable.name.split(':')[0]\n",
    "        print(name)\n",
    "        if \"kernel\" in name and (\"ff\" in name or \"summary\" in name or \"logit\" in name):\n",
    "            print(\"Transposing\")\n",
    "            array = mapping[name].T\n",
    "        elif isinstance(mapping[name], list):\n",
    "            print(mapping[name][0].shape)\n",
    "            c = [torch.unsqueeze(t, 0) for t in mapping[name]]\n",
    "            print(c[0].shape)\n",
    "            array = torch.cat(c, 0)\n",
    "        else:\n",
    "            array = mapping[name]\n",
    "            \n",
    "        variable.load(array.detach().cpu().numpy())\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/husein/parsing/self-attentive-parser/xlnet.py:70: The name tf.gfile.Open is deprecated. Please use tf.io.gfile.GFile instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/parsing/self-attentive-parser/xlnet.py:253: The name tf.variable_scope is deprecated. Please use tf.compat.v1.variable_scope instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/parsing/self-attentive-parser/xlnet.py:253: The name tf.AUTO_REUSE is deprecated. Please use tf.compat.v1.AUTO_REUSE instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/parsing/self-attentive-parser/custom_modeling.py:697: The name tf.logging.info is deprecated. Please use tf.compat.v1.logging.info instead.\n",
      "\n",
      "INFO:tensorflow:memory input None\n",
      "INFO:tensorflow:Use float type <dtype: 'float32'>\n",
      "WARNING:tensorflow:From /home/husein/parsing/self-attentive-parser/custom_modeling.py:704: The name tf.get_variable is deprecated. Please use tf.compat.v1.get_variable instead.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/parsing/self-attentive-parser/custom_modeling.py:809: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.dropout instead.\n",
      "WARNING:tensorflow:From /home/husein/.local/lib/python3.6/site-packages/tensorflow_core/python/layers/core.py:271: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `layer.__call__` method instead.\n",
      "WARNING:tensorflow:\n",
      "The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "  * https://github.com/tensorflow/io (for I/O related ops)\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/husein/parsing/self-attentive-parser/custom_modeling.py:109: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use keras.layers.Dense instead.\n",
      "['model/transformer/r_w_bias:0', 'model/transformer/r_r_bias:0', 'model/transformer/word_embedding/lookup_table:0', 'model/transformer/word_embedding/lookup_table_2:0', 'model/transformer/r_s_bias:0', 'model/transformer/seg_embed:0', 'model/transformer/layer_shared/rel_attn/q/kernel:0', 'model/transformer/layer_shared/rel_attn/k/kernel:0', 'model/transformer/layer_shared/rel_attn/v/kernel:0', 'model/transformer/layer_shared/rel_attn/r/kernel:0', 'model/transformer/layer_shared/rel_attn/o/kernel:0', 'model/transformer/layer_shared/rel_attn/LayerNorm/beta:0', 'model/transformer/layer_shared/rel_attn/LayerNorm/gamma:0', 'model/transformer/layer_shared/ff/layer_1/kernel:0', 'model/transformer/layer_shared/ff/layer_1/bias:0', 'model/transformer/layer_shared/ff/layer_2/kernel:0', 'model/transformer/layer_shared/ff/layer_2/bias:0', 'model/transformer/layer_shared/ff/LayerNorm/beta:0', 'model/transformer/layer_shared/ff/LayerNorm/gamma:0']\n",
      "model/transformer/r_w_bias\n",
      "torch.Size([12, 64])\n",
      "torch.Size([1, 12, 64])\n",
      "WARNING:tensorflow:From <ipython-input-14-2976469a1efd>:79: Variable.load (from tensorflow.python.ops.variables) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Prefer Variable.assign which has equivalent behavior in 2.X.\n",
      "model/transformer/r_r_bias\n",
      "torch.Size([12, 64])\n",
      "torch.Size([1, 12, 64])\n",
      "model/transformer/word_embedding/lookup_table\n",
      "model/transformer/word_embedding/lookup_table_2\n",
      "model/transformer/r_s_bias\n",
      "torch.Size([12, 64])\n",
      "torch.Size([1, 12, 64])\n",
      "model/transformer/seg_embed\n",
      "torch.Size([2, 12, 64])\n",
      "torch.Size([1, 2, 12, 64])\n",
      "model/transformer/layer_shared/rel_attn/q/kernel\n",
      "model/transformer/layer_shared/rel_attn/k/kernel\n",
      "model/transformer/layer_shared/rel_attn/v/kernel\n",
      "model/transformer/layer_shared/rel_attn/r/kernel\n",
      "model/transformer/layer_shared/rel_attn/o/kernel\n",
      "model/transformer/layer_shared/rel_attn/LayerNorm/beta\n",
      "model/transformer/layer_shared/rel_attn/LayerNorm/gamma\n",
      "model/transformer/layer_shared/ff/layer_1/kernel\n",
      "Transposing\n",
      "model/transformer/layer_shared/ff/layer_1/bias\n",
      "model/transformer/layer_shared/ff/layer_2/kernel\n",
      "Transposing\n",
      "model/transformer/layer_shared/ff/layer_2/bias\n",
      "model/transformer/layer_shared/ff/LayerNorm/beta\n",
      "model/transformer/layer_shared/ff/LayerNorm/gamma\n",
      "Tensor(\"model/transformer/dropout_2/Identity:0\", shape=(?, ?, 768), dtype=float32)\n",
      "WARNING:tensorflow:From <ipython-input-14-2976469a1efd>:14: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
      "WARNING:tensorflow:From <ipython-input-14-2976469a1efd>:14: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n",
      "WARNING:tensorflow:From <ipython-input-12-9bba20c7f0a3>:46: to_float (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.cast` instead.\n"
     ]
    }
   ],
   "source": [
    "the_inp_tokens, the_inp_mask, the_out_chart, the_out_tags, bert_features = make_network()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from parse_nk import BERT_TOKEN_MAPPING\n",
    "def bertify_batch(sentences):\n",
    "    all_input_ids = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "    all_input_mask = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "    all_word_end_mask = np.zeros((len(sentences), parser.bert_max_len), dtype=int)\n",
    "\n",
    "    subword_max_len = 0\n",
    "    for snum, sentence in enumerate(sentences):\n",
    "        tokens = []\n",
    "        word_end_mask = []\n",
    "\n",
    "        cleaned_words = []\n",
    "        for word in sentence:\n",
    "            word = BERT_TOKEN_MAPPING.get(word, word)\n",
    "            if word == \"n't\" and cleaned_words:\n",
    "                cleaned_words[-1] = cleaned_words[-1] + \"n\"\n",
    "                word = \"'t\"\n",
    "            cleaned_words.append(word)\n",
    "\n",
    "        for word in cleaned_words:\n",
    "            word_tokens = parser.bert_tokenizer.tokenize(word)\n",
    "            for _ in range(len(word_tokens)):\n",
    "                word_end_mask.append(0)\n",
    "            word_end_mask[-1] = 1\n",
    "            tokens.extend(word_tokens)\n",
    "        tokens.append(\"<sep>\")\n",
    "        word_end_mask.append(1)\n",
    "        tokens.append(\"<cls>\")\n",
    "        word_end_mask.append(1)\n",
    "\n",
    "        input_ids = parser.bert_tokenizer.convert_tokens_to_ids(tokens)\n",
    "        input_mask = [1] * len(input_ids)\n",
    "\n",
    "        subword_max_len = max(subword_max_len, len(input_ids))\n",
    "\n",
    "        all_input_ids[snum, :len(input_ids)] = input_ids\n",
    "        all_word_end_mask[snum, :len(word_end_mask)] = word_end_mask\n",
    "        all_input_mask[snum, :len(input_mask)] = input_mask\n",
    "\n",
    "    all_input_ids = all_input_ids[:, :subword_max_len]\n",
    "    all_word_end_mask = all_word_end_mask[:, :subword_max_len]\n",
    "    all_input_mask = all_input_mask[:, :subword_max_len]\n",
    "    return all_input_ids, all_word_end_mask, all_input_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2, 39, 768)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subbatch_trees = test_treebank[:2]\n",
    "subbatch_sentences = [[leaf.word for leaf in tree.leaves()] for tree in subbatch_trees]\n",
    "inp_val_tokens, inp_val_mask, input_mask = bertify_batch([[word for word in sentence] for sentence in subbatch_sentences])\n",
    "sess.run(bert_features, {the_inp_tokens: inp_val_tokens, the_inp_mask: input_mask}).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 28/28 [00:03<00:00,  8.58it/s]\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "eval_batch_size = 16\n",
    "test_predicted = []\n",
    "for start_index in tqdm(range(0, len(test_treebank), eval_batch_size)):\n",
    "    subbatch_trees = test_treebank[start_index:start_index+eval_batch_size]\n",
    "    subbatch_sentences = [[leaf.word for leaf in tree.leaves()] for tree in subbatch_trees]\n",
    "    inp_val_tokens, inp_val_mask, _ = bertify_batch([[word for word in sentence] for sentence in subbatch_sentences])\n",
    "    out_val_chart, out_val_tags = sess.run((the_out_chart, the_out_tags), \n",
    "                                       {the_inp_tokens: inp_val_tokens, the_inp_mask: inp_val_mask})\n",
    "    trees = []\n",
    "    scores = []\n",
    "    for snum, sentence in enumerate(subbatch_sentences):\n",
    "        chart_size = len(sentence) + 1\n",
    "        tf_chart = out_val_chart[snum,:chart_size,:chart_size,:]\n",
    "        sentence = list(zip([TAG_VOCAB[idx] for idx in out_val_tags[snum,1:chart_size]], [x for x in sentence]))\n",
    "        tree, score = parser.decode_from_chart(sentence, tf_chart)\n",
    "        trees.append(tree)\n",
    "        scores.append(score)\n",
    "    test_predicted.extend([p.convert() for p in trees])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<trees_newline.InternalTreebankNode object at 0x7fd1bd0755f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06d52e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bd0962e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06d5908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bd02bef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06d5fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bd04ab70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06d6668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bd05cf60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06d6cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc79c860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06da390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc7a0160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06daa58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc7a2ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06dd0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc7265f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06dd7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc72a240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ddeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc72ce10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06e08d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc730a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06e2358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc7345c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06e2c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc738240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06e66a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc739e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06e80b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc73e860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06e8a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc7434a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06d0828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc744668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06f58d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc749358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06f5d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc749da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06f44a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc74d5f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06f47b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc74e518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06f4d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc750438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06c9160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc753358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06c9550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc754278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06c9940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc756160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06c9d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc759048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06c8160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc759f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06c8550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc75be48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06c8940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc75f358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06c8d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc75fe48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06cd2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc761940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06cd550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc762438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ce9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc762f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06cec88>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc765a20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06cef28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6e6588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d063f208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6e8048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d063f4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6e8b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d063f7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6ecba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d063fa58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6f0be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0641320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6f4cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0641ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6f7eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0644550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6fa0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06449b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6fc2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0644eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6fe518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0647390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc7006d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06477f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc701898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0647c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc704a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d064b0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc705cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d064f3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc708c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d064f828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc70e898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d064e0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc712860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d064ee10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc713860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06522e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc715860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06527f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc717898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0652cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc719828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0655198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc71a7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0655630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc71d828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0655b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc720860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0655fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc724390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06584a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc725eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0658b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6a75f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d065b208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6ab5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d065b3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6abb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d065bcc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6ad0b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06cdd30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6ad5c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ce0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6adac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ce630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6b0048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ce780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6b05f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ce8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6b0b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ce240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6b20f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ce208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6b56a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ce390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6b97b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d064be80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6bb240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d065bef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6bbcc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ec128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6bd748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ec3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6bf1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ec668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6bfc18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ec940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6c06a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ecbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6c3128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06ece80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6c3b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066a550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6c6160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066a7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6c69b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066acf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6ca240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066af28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6cab00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066b198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6c9400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066b438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6c9c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066b668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6cc550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066b898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6cce10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066bac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6ce710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066bcf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6d1e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066bf28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6d7550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066e4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6d9c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d066eeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6e1208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0672860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6e4860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0675208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc668f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0675be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6705c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0603b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc671c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0605550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6762b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0605ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc677898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06091d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc679f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0609898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc67d630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0609fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc67fd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d060d780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc685400> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d060df98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc686a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d060f668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc68a208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d060fdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc68e9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0612668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc694438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0612fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc699f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0616908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc69f978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0618358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6a44a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0618d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc627f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0619898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc62ae80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0601198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc62cd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06015f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc62ec18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0601978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc630b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0601d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6319e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0603278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6348d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06035f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6367b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0603940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6386a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d061c668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc639fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d061c978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc63e908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d061cf60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc642240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05fb588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc643ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05fbb70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc648470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0621198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc64ada0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0621780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc64d748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0621d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6510f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0624358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc651748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06eabe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc651da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06eadd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc653438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0677cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc653a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0677c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6550b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0677e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6556d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0677b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc655d30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0677748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6573c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06779b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc65a588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0677860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc65e128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0624ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc65f828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0627358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5e6dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06278d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5ead30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06295c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5ec2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0629d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5ec7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0629eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5ecd30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062b048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5ee278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062b828>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5ee780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062b978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5eecc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062bac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5f0208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062bc18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5f0748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062bd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5f6208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062beb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5f7160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062e9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5f9128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062edd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5f9fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0633278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5fbef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06336a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5fde80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0633ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc600dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0633ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc602cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0636358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc604c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06367b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6084e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0636be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc609e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05ba2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc60c7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05c2a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc610320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05c5128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc613c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05c5780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6176d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05c5e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc61b198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05c84a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc61cb00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05c8b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc622048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05cb320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc6249e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05cbc50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5a8390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05ce400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5a9d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05ceb38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5af780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05d22b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5b2160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05d29b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5b3b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05d5128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5b8518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05d57f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5b9ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05d5f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5bf470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05d76a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5c4940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05ddba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5c7ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d061e550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5ce470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d061ee10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5d1978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05bc748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5d5f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05df048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5db550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05df898>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5deb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05e2208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5e3128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05e2a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5e5780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05e2fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc567d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05e5518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc56b320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05e5a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc56d978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05e5ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc56ff60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05e8470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc572550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05e8940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc575b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05e8e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc578240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05eb3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc57b048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05f8668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc57bdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05f8a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc57dbe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05f8e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc57e9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d057b208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5817f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d057b630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc583630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d057ba20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc585470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d057bdd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc58ab70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d057e208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5932e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0580128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5999b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0585048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5a1128> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0585ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc528908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0588e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc52df98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0666e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc536710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d058ac50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc53ce48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d058ea58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc543a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0590978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5496a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05ddb70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc54f278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d064b550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc553e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05db2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc55aa20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05dbda0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5605f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05f43c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4e61d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0593588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4ebdd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0597240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4ee908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0597c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4f0470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0597f60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4f0f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d059b2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4f2b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d059b5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4f36a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d059b8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4f7208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d059bbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4f7cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d059bef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4f9898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d059e240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4fc518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d059e550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc502160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05acbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc503da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a82e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc507a58> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a8a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc50c710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05b2160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5102e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05b2860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc512fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05b2fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc516c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05b56a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc51b8d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05b5d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc51f7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05b84a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc5236d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05b8ba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4aaf98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d053c2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4b2780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d053f2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4b8fd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0542320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4c07b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05452e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4c7f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0547358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4d17b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d054b358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4d6f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d054e780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4e0780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a2860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4e1f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a6710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc467908> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a6e48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc46f2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a5780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc472c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05510f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4794a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0551a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc47dda0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0555358>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc483710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0555c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc486a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05595f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc489e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05599b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc48c198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0559d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc48e588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055c160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc491748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055c518>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc492940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055c908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc494a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055ccf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc496c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a1208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc498dd8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a15f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc49bf98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a19e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4a40f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05a1dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc42b668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0568e80>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc432c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d056bc18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc43b1d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d056f940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc440710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0572668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc447da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05743c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc44f470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05780f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc453ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0578dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc459f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04fb780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc45df60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04fd400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc4637f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04fdbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3e60f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0500198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3e8978> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05006d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3ec278> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062f5f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3efac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062f860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3f3358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05c2908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3f4c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05c2978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3f94e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d059ee48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3fb9b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d062b470>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3fef60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055e3c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc401470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055e940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc404940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055eeb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc406e48> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0561128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc40b358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0561668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc40c7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0561c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc40ed30> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0505198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc413cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0505710>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc416cc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0505e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc41ac88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0508550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc41fc88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d050acc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc422c50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d050e400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3a8c18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d050eb00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3abb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0511240>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3afb38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0511978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3b49b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05130b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3b6f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0513860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3bb240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0513d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3bbfd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05172e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3c1940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0517a58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3c5630> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0517f28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3ca358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d051b630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3cd048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d051bd30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3cfcc0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d051e4e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3d49e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d051eba8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3d8710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05202e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3daef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05bac50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3df748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d050a6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3e0f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d050a0b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc366780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0520cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc367f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0526160>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc36d780> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0526780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc371048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0526da0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc372828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0529400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc374d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0529a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3792e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0529ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc37b898> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d052c400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc37ce80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d052c940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3813c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d052ce10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc383940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d052f320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc385f60> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d052f7f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3895c0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d052fcc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc38e860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0533b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc392b00> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05362e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc396da0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0536a20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc39afd0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05381d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3a2240> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0538940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3a4390> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04bc128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc32a198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04bc5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc32b6a0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04bf0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc32eba8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04bf5c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc332080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04bfa58>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc333518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04bfef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3359b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04c13c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc339eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04c1860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc33c470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04c1cf8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc33e9e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04c4208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc341b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04c46a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc346cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0533278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc34ae10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04c85c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc34ef28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04c8dd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc355080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0524630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc359198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0524e10>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc35d2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04cd630>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc362470> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04cde48>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc363828> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d06a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2e6be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d0ac8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2e8f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d0ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2ed358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d43c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2ee710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d47f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2f0a90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d4c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2f1e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d6080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2f71d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d64e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2f8be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d6940>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2fccf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04dd9b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc301f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04e2278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc307208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04e2a90>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc30c4a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04e52e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3107b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04e5b38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc313940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04e9390>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc318be0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04e9b70>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc31bef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04eb320>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc3225f8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04ebb38>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2a8198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04ee438>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2ad7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04eeef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2b0eb8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04f1860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2b6550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04f41d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2bbc50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04f4b00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2c12e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04f7400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2c59b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04f7d68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2c8e10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055f0f0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2cb2b0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055f6d8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2cd748> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d055fbe0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2d0b70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04dd198>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2d2f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04dd6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2d6438> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04da278>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2d87f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04da7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2dac50> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04dada0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2de0f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d047aa20>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2dfbe0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d9048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2664e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04d9748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc26ad68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0482208>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc271320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0482be0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2734e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04853c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc278080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0485860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc27a588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0485eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc27ad68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0601080>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc27b588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05baf98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc27bd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05bab00>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc27c588> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d054e550>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc27cd68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0533978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc280518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0533780>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc280cf8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05339e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc281518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05334a8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc287f98> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05331d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc28cb70> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04886a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc28ef28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d0488c18>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc292b38> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d048d128>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2977b8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d048d748>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc29b3c8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d048dd68>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2a0048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04903c8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2a2c88> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04909b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2a5860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d047ddd8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc22a4e0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d049b400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc230160> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d049b9e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2332e8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d049e048>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc238358> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d049e7b8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc23c4a8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d049ef28>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc240518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04a06a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc244550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04a0eb8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc249668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04a35f8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc24d7f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04a3d30>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc24ea90> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04a74e0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc252e80> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04a79e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc256320> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04a7ef0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2586d8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04aa400>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc25ba20> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04aa8d0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc25ce10> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04aada0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc2611d0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d047d978>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc262550> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d047d0b8>\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc1e9198> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d047d588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc1eb860> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04ac908>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc1f0048> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04acf60>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc1f27f0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04b05c0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc1f4f28> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04b0c50>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc1f7710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04b32e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc1faef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04b39b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc1ff710> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04b3fd0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc202ef0> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04b5668>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc203ac8> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04b5cc0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc205668> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d04b5f98>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc207208> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d048b2b0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc207d68> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d048b588>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc209940> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d048b860>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc20a518> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05ac2e8>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc20c080> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d05ac6a0>\n",
      "<trees_newline.InternalTreebankNode object at 0x7fd1bc20cc18> <class 'trees_newline.InternalTreebankNode'> <trees_newline.InternalTreebankNode object at 0x7fd0d06cda90>\n"
     ]
    }
   ],
   "source": [
    "test_fscore = evaluate.evalb('EVALB/', test_treebank[:len(test_predicted)], test_predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'(Recall=1.67, Precision=10.00, FScore=2.86, CompleteMatch=0.00, TaggingAccuracy=28.99)'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(test_fscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_node_names = [the_inp_tokens.name.split(':')[0], the_inp_mask.name.split(':')[0]]\n",
    "output_node_names = [the_out_chart.name.split(':')[0], the_out_tags.name.split(':')[0]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "graph_def = tf.graph_util.convert_variables_to_constants(sess, sess.graph_def, output_node_names)\n",
    "from tensorflow.tools.graph_transforms import TransformGraph\n",
    "graph_def = TransformGraph(graph_def, input_node_names, output_node_names, [\n",
    "'strip_unused_nodes()',\n",
    "'remove_nodes(op=Identity, op=CheckNumerics)',\n",
    "'fold_constants()',\n",
    "'fold_old_batch_norms',\n",
    "'fold_batch_norms',\n",
    "'round_weights(num_steps=128)',\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('export/xlnet-base.pb', 'wb') as f:\n",
    "    f.write(graph_def.SerializeToString())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "with open('vocab-xlnet-base.json', 'w') as fopen:\n",
    "    json.dump({'label': LABEL_VOCAB, 'tag': TAG_VOCAB}, fopen)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
